{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "from detail import Detail\n",
    "import numpy as np\n",
    "import skimage.io as io\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import pylab\n",
    "import json\n",
    "import random\n",
    "pylab.rcParams['figure.figsize'] = (10.0, 8.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "annFile='../../json/PASCAL_trainval_merged.json' # annotations\n",
    "imgDir='../../VOCdevkit/VOC2010/JPEGImages' # jpeg images\n",
    "phase='trainval'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=7.37s)\n",
      "creating index...\n",
      "index created! 6.75s\n",
      "\n",
      "***info***\n",
      "description: This is the 3.0 version of the CVPR 2017 PASCAL in Detail dataset.\n",
      "version: 3.0\n",
      "year: 2017\n",
      "contributor: https://sites.google.com/view/pasd\n",
      "date_created: 2017-07-18\n"
     ]
    }
   ],
   "source": [
    "# initialize detail api for instance annotations\n",
    "detailsGt = Detail(annFile, imgDir, phase)\n",
    "\n",
    "print('\\n***info***')\n",
    "detailsGt.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load evaluation module for image classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total category #59, total img #10100 \n"
     ]
    }
   ],
   "source": [
    "from detail import detaileval_cls \n",
    "from importlib import reload\n",
    "reload(detaileval_cls)\n",
    "detaileval = detaileval_cls.DetailEvalCls(detailsGt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fake result json for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## fake a result json for evaluation\n",
    "detailRes = []\n",
    "count = 0\n",
    "for gt_img in detaileval.gt_imgs:\n",
    "    detailRes.append({'image_id': gt_img['image_id'], 'category_score': np.ndarray.tolist(np.zeros(59))})\n",
    "    if(2 in gt_img['categories']):\n",
    "        detailRes[-1]['category_score'][0] = 1\n",
    "    if(23 in gt_img['categories'] and count < 200):\n",
    "        count += 1\n",
    "        detailRes[-1]['category_score'][1] = 1\n",
    "    '''\n",
    "    for k in gt_img['categories']:\n",
    "        cat_idx = detaileval.cat_idxs[k]\n",
    "        detailRes[-1]['category_score'][cat_idx] = 1\n",
    "    '''\n",
    "result = {}\n",
    "result['pred'] = detailRes\n",
    "detail_res_file = '../res/detail_cls_fake_result.json'\n",
    "json.dump(result, open(detail_res_file, 'w'), indent = 4)\n",
    "\n",
    "detailRes = json.load(open('../res/detail_cls_fake_result.json', 'r'))['pred']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluate annotation type cls\n",
      "{'category_id': 2, 'name': 'aeroplane', 'ap': 1.0}\n",
      "{'category_id': 23, 'name': 'bicycle', 'ap': 0.53237909908101999}\n",
      "{'category_id': 25, 'name': 'bird', 'ap': 0.067996680116887975}\n",
      "{'category_id': 31, 'name': 'boat', 'ap': 0.043053950463692578}\n",
      "{'category_id': 34, 'name': 'bottle', 'ap': 0.061904164575358474}\n",
      "{'category_id': 45, 'name': 'bus', 'ap': 0.046632527524791287}\n",
      "{'category_id': 59, 'name': 'car', 'ap': 0.10270417895347403}\n",
      "{'category_id': 65, 'name': 'cat', 'ap': 0.11623565408709849}\n",
      "{'category_id': 72, 'name': 'chair', 'ap': 0.10793164449217801}\n",
      "{'category_id': 98, 'name': 'cow', 'ap': 0.030388715944581562}\n",
      "{'category_id': 397, 'name': 'diningtable', 'ap': 0.056767440231770362}\n",
      "{'category_id': 113, 'name': 'dog', 'ap': 0.13950492488909666}\n",
      "{'category_id': 207, 'name': 'horse', 'ap': 0.043407806625990732}\n",
      "{'category_id': 258, 'name': 'motorbike', 'ap': 0.047173460352343571}\n",
      "{'category_id': 284, 'name': 'person', 'ap': 0.36780788155087263}\n",
      "{'category_id': 308, 'name': 'pottedplant', 'ap': 0.052724812150599362}\n",
      "{'category_id': 347, 'name': 'sheep', 'ap': 0.038742754914273203}\n",
      "{'category_id': 368, 'name': 'sofa', 'ap': 0.052327905895205515}\n",
      "{'category_id': 416, 'name': 'train', 'ap': 0.057239435946436018}\n",
      "{'category_id': 427, 'name': 'tvmonitor', 'ap': 0.052866266733750153}\n",
      "{'category_id': 9, 'name': 'bag', 'ap': 0.035871031064712317}\n",
      "{'category_id': 18, 'name': 'bed', 'ap': 0.01395544961341097}\n",
      "{'category_id': 22, 'name': 'bench', 'ap': 0.013327998949186281}\n",
      "{'category_id': 33, 'name': 'book', 'ap': 0.044815191618757085}\n",
      "{'category_id': 44, 'name': 'building', 'ap': 0.3053451715811536}\n",
      "{'category_id': 46, 'name': 'cabinet', 'ap': 0.055880497864523274}\n",
      "{'category_id': 68, 'name': 'ceiling', 'ap': 0.061395884675105504}\n",
      "{'category_id': 80, 'name': 'cloth', 'ap': 0.088789085082249211}\n",
      "{'category_id': 85, 'name': 'computer', 'ap': 0.012544853646827265}\n",
      "{'category_id': 104, 'name': 'cup', 'ap': 0.04630896258508059}\n",
      "{'category_id': 115, 'name': 'door', 'ap': 0.053093509108217286}\n",
      "{'category_id': 144, 'name': 'fence', 'ap': 0.11339128060909517}\n",
      "{'category_id': 158, 'name': 'floor', 'ap': 0.19406704009416273}\n",
      "{'category_id': 159, 'name': 'flower', 'ap': 0.019622898139060409}\n",
      "{'category_id': 162, 'name': 'food', 'ap': 0.029032282398863903}\n",
      "{'category_id': 187, 'name': 'grass', 'ap': 0.28225444711647074}\n",
      "{'category_id': 189, 'name': 'ground', 'ap': 0.30509091706188918}\n",
      "{'category_id': 220, 'name': 'keyboard', 'ap': 0.016712871260000949}\n",
      "{'category_id': 232, 'name': 'light', 'ap': 0.10361827210525285}\n",
      "{'category_id': 259, 'name': 'mountain', 'ap': 0.072602616558633523}\n",
      "{'category_id': 260, 'name': 'mouse', 'ap': 0.015006234108723905}\n",
      "{'category_id': 105, 'name': 'curtain', 'ap': 0.04106389334203675}\n",
      "{'category_id': 296, 'name': 'platform', 'ap': 0.024183018380288947}\n",
      "{'category_id': 355, 'name': 'sign', 'ap': 0.064477160156055285}\n",
      "{'category_id': 295, 'name': 'plate', 'ap': 0.03584916987715539}\n",
      "{'category_id': 324, 'name': 'road', 'ap': 0.11572132010904598}\n",
      "{'category_id': 326, 'name': 'rock', 'ap': 0.040364342233022849}\n",
      "{'category_id': 349, 'name': 'shelves', 'ap': 0.054637769787487488}\n",
      "{'category_id': 354, 'name': 'sidewalk', 'ap': 0.040449437683559589}\n",
      "{'category_id': 360, 'name': 'sky', 'ap': 0.32636567577040454}\n",
      "{'category_id': 366, 'name': 'snow', 'ap': 0.018026206154579928}\n",
      "{'category_id': 19, 'name': 'bedclothes', 'ap': 0.042811811161293151}\n",
      "{'category_id': 415, 'name': 'track', 'ap': 0.047175112558621765}\n",
      "{'category_id': 420, 'name': 'tree', 'ap': 0.32088137234735936}\n",
      "{'category_id': 424, 'name': 'truck', 'ap': 0.0128069215703172}\n",
      "{'category_id': 440, 'name': 'wall', 'ap': 0.29200414853830692}\n",
      "{'category_id': 445, 'name': 'water', 'ap': 0.090191623057455761}\n",
      "{'category_id': 454, 'name': 'window', 'ap': 0.08355303807455762}\n",
      "{'category_id': 458, 'name': 'wood', 'ap': 0.049257652409394545}\n",
      "======= \n",
      " Mean ap over #59 categories 0.111870\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.1118700927624024"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# start evaluation\n",
    "detaileval.loadRes(detailRes)\n",
    "detaileval.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py36-test",
   "language": "python",
   "name": "py36-test"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
